/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */

package utfpr.cp.crawler;

import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.parser.HtmlParseData;
import edu.uci.ics.crawler4j.parser.TextParseData;
import edu.uci.ics.crawler4j.url.WebURL;
import java.util.ArrayList;
import java.util.List;
import org.apache.http.HttpStatus;
import utfpr.cp.control.ResourceControl;
import utfpr.cp.control.URLControl;
import utfpr.cp.model.URLData;

/**
 *
 * @author Adriano_2
 */
public class PageDownloader extends WebCrawler {
    ThreadManager manager;
    
    @Override
    public void onStart() {
        manager = (ThreadManager) myController.getCustomData();
    }
    
    @Override
    public boolean shouldVisit(WebURL url) {
        manager.getStats().addTotalProcessedURL();
        if (!manager.shoudVisit(url)) {
            return false;
        }
        
        if (ResourceControl.existResource(url.getURL())) {
            manager.addBlockedPage();
            return manager.getBlockedPage() > 10;
        }
        
        return true;
    }
    
    @Override
    public void visit(Page page) {
        String url = page.getWebURL().getURL();
        manager.getStats().addTotalVisitedPages();
        manager.resetBlockedPage();
        URLData ownerURL = URLControl.getOrCreate(page.getWebURL());
        
        if (ResourceControl.existResource(ownerURL.getUrl())) {
            Util.showDebug("Skipping: " + url);
            return;
        }
        
        String content;
        List<WebURL> links;
        if (page.getParseData() instanceof HtmlParseData) {
            HtmlParseData htmlParseData = (HtmlParseData) page.getParseData();
            content = htmlParseData.getHtml();
            links = htmlParseData.getOutgoingUrls();
        } else if (page.getParseData() instanceof TextParseData) {
            TextParseData txt = (TextParseData) page.getParseData();
            content = txt.getTextContent();
            links = new ArrayList<>();
        }
        else {
            Util.showDebug("Invalid parser: " + url);
            return;
        }
        manager.getStats().addTotalBytes(content.getBytes().length);
        if (ResourceControl.create(ownerURL, content, links, page.getFetchResponseHeaders(), manager.getTarget())) {
            Util.showDebug("OK: " + url);
            manager.getStats().addTotalSavedPages();
        }
        else {
            Util.showDebug("DBError: " + url);
            manager.getStats().addDatabaseErrors();
        }
    }
    
    
    @Override
    protected void handlePageStatusCode(WebURL webUrl, int statusCode, String statusDescription) {                
        if (statusCode != HttpStatus.SC_OK) {
            Util.showDebug("HTTP Error ("+statusDescription + "):" + webUrl.getURL());
            manager.getStats().addErrorPages();
        }
    }
    
    @Override
    public void onContentFetchError(WebURL webUrl) {
        
    }
    
    @Override
    public void onParseError(WebURL webUrl) {
        
    }
    
    @Override
    public void onBeforeExit() {
            
    }
}
